#!/bin/env bash
get_real_path(){
  if [ "${1:0:1}" == "/" ]; then
    echo "$1"
  else
    echo "$(realpath -m $PWD/$1)"
  fi
}

get_datetime(){
  date +%Y%m%d%H%M%S
}
MS_HCCL_CM_INIT=0
# set the environment for root
export PATH=/usr/local/python3.7.5/bin:$PATH
export LD_LIBRARY_PATH=/usr/local/python3.7.5/lib:$LD_LIBRARY_PATH
source /usr/local/Ascend/ascend-toolkit/set_env.sh

CURRENT_DIR=$(pwd)
SOURCECODE_DIR="/home/liupan/WP/WP-GeneralPurposeMedicalFoundationModels/scripts"
cd ${SOURCECODE_DIR} || exit

execute_path=/home/liupan/WP/WP-GeneralPurposeMedicalFoundationModels
echo ${execute_path}

export DEPLOY_MODE=0
export CONFIG_PATH=/home/liupan/WP/WP-GeneralPurposeMedicalFoundationModels/simplified/config/ttvitbase.yaml
export EXP_TYPE="mm"
export EXP_NAME="ttvitbaseultractawithDeJi"
export RANK_SIZE=16
export RANK_START=8
export RANK_TABLE_FILE=/home/liupan/WP/WP-GeneralPurposeMedicalFoundationModels/hccl/test_config.json

export TEMP_PATH=/data/liupan/temp/startrank${RANK_START}
mkdir -p $TEMP_PATH
export LOCAL_DEFAULT_PATH=$TEMP_PATH


DEVICE_START=0
DEVICE_SIZE=8

source /usr/local/Ascend/driver/bin/setenv.bash
rm -rf $TEMP_PATH/WP-GeneralPurposeMedicalFoundationModels
rsync -rv --exclude-from=/home/liupan/WP/WP-GeneralPurposeMedicalFoundationModels/.copyignore $execute_path $TEMP_PATH
cd $TEMP_PATH/WP-GeneralPurposeMedicalFoundationModels || exit

export LOG_PATH=/data1/pretrain/log/${EXP_TYPE}_${EXP_NAME}
mkdir -p $LOG_PATH

for((i=DEVICE_START;i<DEVICE_SIZE;i++));
do
  export RANK_ID=$[i+RANK_START]
  export DEVICE_ID=$[i+DEVICE_START]
  echo "start training for rank $RANK_ID, device $DEVICE_ID"
  python3 $TEMP_PATH/WP-GeneralPurposeMedicalFoundationModels/simplified/train_examine.py --config $CONFIG_PATH --exp_type $EXP_TYPE --exp_name $EXP_NAME --output_dir $LOG_PATH 2>&1 | tee $LOG_PATH/print_rank_${RANK_ID}.log &
done
cd $execute_path || exit
